
import pandas as pd
import numpy as np

from sklearn import metrics
from sklearn.model_selection import cross_val_score, KFold, StratifiedShuffleSplit, ShuffleSplit

# Pytorch
import torch
from torch.autograd import Variable
import torch.nn as nn 
import torchvision.transforms as transforms

# Visualisation libraries

## Progress Bar
import progressbar

## Text
from colorama import Fore, Back, Style
from IPython.display import Image, display, Markdown, Latex, clear_output

## plotly
from plotly.offline import init_notebook_mode, iplot 
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px

## seaborn
import seaborn as sns
sns.set_style("whitegrid")
sns.set_context("paper", rc={"font.size":12,"axes.titlesize":14,"axes.labelsize":12})

## matplotlib
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
from matplotlib.patches import Ellipse, Polygon
import matplotlib.gridspec as gridspec
import matplotlib.colors
from pylab import rcParams
from matplotlib.font_manager import FontProperties
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
plt.style.use('seaborn-whitegrid')
import matplotlib as mpl
mpl.rcParams['figure.figsize'] = (17, 6)
mpl.rcParams['axes.labelsize'] = 14
mpl.rcParams['xtick.labelsize'] = 12
mpl.rcParams['ytick.labelsize'] = 12
mpl.rcParams['text.color'] = 'k'
%matplotlib inline

import warnings
warnings.filterwarnings("ignore")


# loading data
train_data = pd.read_csv("digit_recognizer/train.csv", dtype = np.float32)
display(train_data.head())


# y set: Labels
y = train_data['label'].values
# X set: Pixels
X = train_data.iloc[:,1:].values/255


fig, ax = plt.subplots(nrows=4, ncols=12, figsize=(12, 6), subplot_kw={'xticks': [], 'yticks': []})
ax = ax.ravel()

font = FontProperties()
font.set_family('fantasy')

n = int(np.sqrt(X[0].shape)[0])
for i in range(len(ax)):
    _ = ax[i].imshow(X[i].reshape(n,n))
    _ = ax[i].set_title('%i' % y[i], fontproperties=font, fontsize = 16)
    _ = ax[i].set_aspect(1)
    
plt.tight_layout(h_pad=.02, w_pad=.2)


def Dist_Plot(y, Plot_dict):
    def ToSeries(x):
        if not isinstance(x, pd.Series):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths= Plot_dict['column_widths'],
                        specs=[[{"type": "table"},{"type": "pie"}]])
    
    # Right
    y = ToSeries(y)
    fig.add_trace(go.Pie(labels= np.sort(y.unique()),
                         values= y.value_counts().values, pull=[0.1],
                         textfont=dict(size=Plot_dict['textfont']),
                         marker=dict(colors = Plot_dict['PieColors'],
                         line=dict(color='black', width=1))), row=1, col=2)
    fig.update_traces(hole=.5)
    fig.update_layout(legend=dict(orientation="v"), legend_title_text= Plot_dict['legend_title'])
    
    # Left
    
    ## Table
    Table = y.value_counts().to_frame('Count').reset_index(drop = False)
    Table = Table.rename(columns = {'index':Plot_dict['legend_title']}).sort_values(by = [Plot_dict['legend_title']])
    Table['Percentage'] = np.round(100*(Table['Count']/Table['Count'].sum()),2)

    T = Table.copy()
    T['Percentage'] = T['Percentage'].map(lambda x: '%% %.2f' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    TableColors = Plot_dict['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.2, 0.4, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
                                     'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
    if not Plot_dict['height'] == None:
        fig.update_layout(height = Plot_dict['height'])
    fig.show()
    
Plot_dict = dict(column_widths = [0.3, 0.7],  PieColors = None, textfont = 12,
                 TableColors = ['Navy','White'], height = 450, legend_title = 'Digits')
Dist_Plot(y, Plot_dict)


Labels = np.unique(y).astype(int)

Test_Size = 0.3
sss = StratifiedShuffleSplit(n_splits=1, test_size=Test_Size, random_state=42)
_ = sss.get_n_splits(X, y)
for train_index, test_index in sss.split(X, y):
    # X
    if isinstance(X, pd.DataFrame):
        X_train, X_test = X.loc[train_index], X.loc[test_index]
    else:
        X_train, X_test = X[train_index], X[test_index]
    # y    
    if isinstance(y, pd.Series):
        y_train, y_test = y[train_index], y[test_index]
    else:
        y_train, y_test = y[train_index], y[test_index]
del sss


def Train_Test_Dist(X_train, y_train, X_test, y_test, Plot_dict):
    def ToSeries(x):
        if not isinstance(x, pd.Series):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=3, horizontal_spacing = 0.02, column_widths= Plot_dict['column_widths'],
                        specs=[[{"type": "table"},{'type':'domain'}, {'type':'domain'}]])
    
    y_train = ToSeries(y_train)
    y_test = ToSeries(y_test)
    # Right
    C = 2
    for y in [y_train, y_test]:
        fig.add_trace(go.Pie(labels= np.sort(y.unique()),
                             values= y.value_counts().values, pull=[0.1],
                             textfont=dict(size=Plot_dict['textfont']),
                             marker=dict(colors = Plot_dict['PieColors'],
                             line=dict(color='black', width=1))), row=1, col=C)
        fig.update_traces(hole=.5)
        fig.update_layout(legend=dict(orientation="v"), legend_title_text= Plot_dict['legend_title'])
        C+=1
    
    # Left
    
    # Table
    Table = pd.DataFrame(data={'Set':['X_train','X_test','y_train','y_test'],
                       'Shape':[X_train.shape, X_test.shape, y_train.shape, y_test.shape]}).astype(str)
    T = Table.copy()
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
        
    TableColors = Plot_dict['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.2, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
                                     'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
    if not Plot_dict['height'] == None:
        fig.update_layout(height = Plot_dict['height'])
    fig.show()
    
Plot_dict = dict(column_widths = [0.3, 0.3, 0.3],  PieColors = None, textfont = 12,
                 TableColors = ['Navy','White'], height = 450, legend_title = 'Digits')
Train_Test_Dist(X_train, y_train, X_test, y_test, Plot_dict)


def Header(Text, L = 100, C = 'Blue', T = 'White'):
    BACK = {'Black': Back.BLACK, 'Red':Back.RED, 'Green':Back.GREEN, 'Yellow': Back.YELLOW, 'Blue': Back.BLUE,
         'Magenta':Back.MAGENTA, 'Cyan': Back.CYAN}
    FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
         'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
    print(BACK[C] + FORE[T] + Style.NORMAL + Text + Style.RESET_ALL + ' ' + FORE[C] +
          Style.NORMAL +  (L- len(Text) - 1)*'=' + Style.RESET_ALL)
    
def Line(L=100, C = 'Blue'):
    FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
         'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
    print(FORE[C] + Style.NORMAL + L*'=' + Style.RESET_ALL)
    
def Search_List(Key, List): return [s for s in List if Key in s]

def TorchSets(Set):
    # Inut: Arrays
    # GPU Cuda
    if isinstance(Set, (pd.DataFrame, pd.Series)):
        Set = Set.values
    if torch.cuda.is_available():
        if Set.ndim==1:
            Out = Variable(torch.from_numpy(Set).type(torch.LongTensor).cuda())
        else:
            Out = Variable(torch.from_numpy(Set).cuda())
    # CPU
    else:
        if Set.ndim==1:
            Out = Variable(torch.from_numpy(Set).type(torch.LongTensor))
        else:
            Out = Variable(torch.from_numpy(Set))
    return Out

# Tensors
X_train_tensor = TorchSets(X_train)
y_train_tensor = TorchSets(y_train)
X_test_tensor = TorchSets(X_test)
y_test_tensor = TorchSets(y_test)
    
Batch_size = 100
iteration_number = int(4e4)

epochs_number = int(iteration_number / (len(X_train) / Batch_size))

# Pytorch train and test sets
Train_set = torch.utils.data.TensorDataset(X_train_tensor, y_train_tensor)
Test_set = torch.utils.data.TensorDataset(X_test_tensor, y_test_tensor)

# data loader
train_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)
test_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)


class LogisticRegressionModel(torch.nn.Module):
    def __init__(self, input_Size, output_Size):
        super(LogisticRegressionModel, self).__init__()
        self.linear = torch.nn.Linear(input_Size, output_Size)
    
    def forward(self, x):
        out = self.linear(x)
        return out
    
def Plot_history(history, Table_Rows = 25, yLim = 2):
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=[0.6, 0.4],
                        specs=[[{"type": "scatter"},{"type": "table"}]])
    # Left
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Loss'].astype(float).values.round(4),
                             line=dict(color='OrangeRed', width= 1.5), name = 'Loss'), 1, 1)
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Accuracy'].astype(float).values,
                             line=dict(color='MidnightBlue', width= 1.5),  name = 'Accuracy'), 1, 1)
    fig.update_layout(legend=dict(x=0, y=1.1, traceorder='reversed', font_size=12),
                  dragmode='select', plot_bgcolor= 'white', height=600, hovermode='closest',
                  legend_orientation='h')
    fig.update_xaxes(range=[history.Iteration.min(), history.Iteration.max()],
                     showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    fig.update_yaxes(range=[0, yLim], showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    # Right
    ind = np.linspace(0, history.shape[0], Table_Rows, endpoint = False).round(0).astype(int)
    ind = np.append(ind, history.index[-1])
    history = history[history.index.isin(ind)]
    T = history.copy()
    T[['Loss','Accuracy']] = T[['Loss','Accuracy']].applymap(lambda x: '%.4e' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    fig.add_trace(go.Table(header=dict(values = list(history.columns), line_color='darkslategray',
                                       fill_color='Navy', align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.4, 0.4, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color=['Lavender', 'white', 'white']),
                                      align=['center', 'center'], font_size=12,height=20)), 1, 2)
    fig.show()


n_Inst = X_train.shape[1]
n_Att = X_train.shape[1]
    
input_Size, output_Size = n_Inst, len(Labels)

# model
model = LogisticRegressionModel(input_Size, output_Size)

# GPU
if torch.cuda.is_available():
    model.cuda()

# Cross Entropy Loss 
CEL= nn.CrossEntropyLoss()

# Optimizer 
learning_rate = 1e-2
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

# Traning the Model
Count = 0
Loss_list = []
Iteration_list = []
Accuracy_list = []
MSE_list = []
MAE_list = []
Steps = 10

Progress_Bar = progressbar.ProgressBar(maxval= int(iteration_number+iteration_number/10),
                                       widgets=[progressbar.Bar('=', '|', '|'), progressbar.Percentage()])
    
# print('---------------------------------------------------------')
for epoch in range(epochs_number):
    for i, (Xtr, ytr) in enumerate(train_loader):
        
        # Variables
        Xtr = Variable(Xtr.view(-1, n_Att))
        ytr = Variable(ytr)
        
        # Set all gradients to zero
        optimizer.zero_grad()
        
        # Forward
        Out = model(Xtr.float())
        
        # loss
        loss = CEL(Out, ytr.long())
        
        # Backward (Calculating the gradients)
        loss.backward()
        
        # Update parameters
        optimizer.step()
        
        Count += 1
        
        del Xtr, ytr
        
        # Predictions
        if Count % Steps == 0:
            # Calculate Accuracy         
            Correct, Total = 0, 0
            # Predictions
            for Xts, yts in test_loader: 
                Xts = Variable(Xts.view(-1, n_Att))
                
                # Forward
                Out = model(Xts.float())
                
                # The maximum value of Out
                Predicted = torch.max(Out.data, 1)[1]
                
                # Total number of yts
                Total += len(yts)
                
                # Total Correct predictions
                Correct += (Predicted == yts).sum()
            del Xts, yts
            # storing loss and iteration
            Loss_list.append(loss.data)
            Iteration_list.append(Count)
            Accuracy_list.append(Correct / float(Total))
            
        Progress_Bar.update(Count)

Progress_Bar.finish()

history = pd.DataFrame({'Iteration': np.array(Iteration_list),
                      'Loss': np.array([x.cpu().data.numpy() for x in Loss_list]),
                      'Accuracy': np.array([x.cpu().data.numpy() for x in Accuracy_list])})
del Loss_list, Iteration_list, Accuracy_list

|=========================================================================|100%


Plot_history(history, Table_Rows = 18, yLim = 1)


def Confusion_Mat(CM_Train, CM_Test, PD, n_splits = 10):
    if n_splits == None:
        Titles = ['Train Set', 'Test Set']
    else:
        Titles = ['Train Set (CV = % i)' % n_splits, 'Test Set (CV = % i)' % n_splits]
    CM = [CM_Train, CM_Test]
    Cmap = ['Greens', 'YlGn','Blues', 'PuBu']
    for i in range(2):
        fig, ax = plt.subplots(1, 2, figsize= PD['FS'])
        fig.suptitle(Titles[i], weight = 'bold', fontsize = 16)
        _ = sns.heatmap(CM[i], annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i], ax = ax[0],
                        linewidths = 0.2, cbar_kws={"shrink": PD['shrink']})
        _ = ax[0].set_title('Confusion Matrix');
        Temp = np.round(CM[i].astype('float') / CM[i].sum(axis=1)[:, np.newaxis], 2)
        _ = sns.heatmap(Temp,
                        annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i+1], ax = ax[1],
                       linewidths = 0.4, vmin=0, vmax=1, cbar_kws={"shrink": PD['shrink']})
        _ = ax[1].set_title('Normalized Confusion Matrix');

        for a in ax:
            _ = a.set_xlabel('Predicted labels')
            _ = a.set_ylabel('True labels'); 
            _ = a.xaxis.set_ticklabels(Labels)
            _ = a.yaxis.set_ticklabels(Labels)
            _ = a.set_aspect(1)


# Train
y_pred = model(X_train_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Train = pd.DataFrame(metrics.classification_report(y_train, y_pred, target_names=Labels, output_dict=True)).T
CM_Train = metrics.confusion_matrix(y_train, y_pred)
# Test
y_pred = model(X_test_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Test = pd.DataFrame(metrics.classification_report(y_test, y_pred, target_names=Labels, output_dict=True)).T
CM_Test = metrics.confusion_matrix(y_test, y_pred)

Reports_Train = Reports_Train.reset_index().rename(columns ={'index': 'Train Set'})
Reports_Test = Reports_Test.reset_index().rename(columns ={'index': 'Test Set'})
                                                 
display(Reports_Train.style.hide_index().set_properties(**{'background-color': 'HoneyDew', 'color': 'Black'}).\
        set_properties(subset=['Train Set'], **{'background-color': 'SeaGreen', 'color': 'White'}))
display(Reports_Test.style.hide_index().set_properties(**{'background-color': 'Azure', 'color': 'Black'}).\
        set_properties(subset=['Test Set'], **{'background-color': 'RoyalBlue', 'color': 'White'}))

PD = dict(FS = (20, 8), annot_kws = 10, shrink = .6)
Confusion_Mat(CM_Train, CM_Test, PD = PD, n_splits = None)


test_data = pd.read_csv("digit_recognizer/test.csv", dtype = np.float32)
test_data = test_data.values/255

if torch.cuda.is_available():
    test_data_tensor = Variable(torch.from_numpy(test_data).cuda())
else:
    test_data_tensor = Variable(torch.from_numpy(test_data))

Pred = model(test_data_tensor).cpu().data.numpy()
Pred_Labels = pd.DataFrame({'ImageId': np.arange(1,test_data.shape[0]+1), 'Label':Pred.argmax(axis=1)})
# Saving the results
Pred_Labels.to_csv('digit_recognizer/digit_recognizer_sub_PyANN.csv', index=False)

Train Set	precision	recall	f1-score	support
0	0.956344	0.969571	0.962912	2892.000000
1	0.952906	0.974992	0.963823	3279.000000
2	0.922426	0.894665	0.908333	2924.000000
3	0.899573	0.899869	0.899721	3046.000000
4	0.921934	0.936491	0.929156	2850.000000
5	0.902772	0.870482	0.886333	2656.000000
6	0.940400	0.958909	0.949564	2896.000000
7	0.938406	0.924700	0.931502	3081.000000
8	0.897012	0.897328	0.897170	2844.000000
9	0.899320	0.901774	0.900545	2932.000000
accuracy	0.923844	0.923844	0.923844	0.923844
macro avg	0.923109	0.922878	0.922906	29400.000000
weighted avg	0.923653	0.923844	0.923662	29400.000000

Test Set	precision	recall	f1-score	support
0	0.959968	0.966935	0.963439	1240.000000
1	0.939602	0.974377	0.956674	1405.000000
2	0.919606	0.894653	0.906958	1253.000000
3	0.895430	0.885824	0.890601	1305.000000
4	0.904571	0.923077	0.913730	1222.000000
5	0.888993	0.836699	0.862053	1139.000000
6	0.940711	0.958904	0.949721	1241.000000
7	0.917479	0.926515	0.921975	1320.000000
8	0.870386	0.870386	0.870386	1219.000000
9	0.885417	0.879777	0.882588	1256.000000
accuracy	0.913095	0.913095	0.913095	0.913095
macro avg	0.912216	0.911715	0.911812	12600.000000
weighted avg	0.912764	0.913095	0.912781	12600.000000

Digit Recognizer Dataset¶

Data Description¶

Train and Test sets¶

Setting up Tensor Arrays¶

Modeling¶

Confusion Matrix¶

Predictions¶

References¶

	label	...
0	1.0	...
1	0.0	...
2	1.0	...
3	4.0	...
4	0.0	...

	label	...
0	1.0	...
1	0.0	...
2	1.0	...
3	4.0	...
4	0.0	...

	label	...
0	1.0	...
1	0.0	...
2	1.0	...
3	4.0	...
4	0.0	...